#!/usr/bin/perl

use DBI;
use strict;
use PGPLOT;
use Time::Local;

# this script looks for RPFITS files in the rpfits_files table, extracts them
# to a MIRIAD dataset, then creates an observation summary plot of that file.

my $staging_area="/data/MENTOK_2/ste616/flux_monitoring/staging_area";

my $dsn="dbi:Pg:dbname=flux_monitoring;host=localhost;port=5432";
my $user="postgres";
my $password="";

# Set up a connection to the database listing the RPFITS files that need to be
# processed.
my $dbh=DBI->connect($dsn,$user,$password,{ RaiseError => 0, AutoCommit => 1});

# Query for all RPFITS files that need summarising
my $sth=$dbh->prepare("SELECT * FROM rpfits_files WHERE summarised=FALSE ORDER BY file LIMIT 1000");
$sth->execute;
my @files_to_process;
my @just_files;
while (my ($rpfits_file,$rpfits_path,$tp)=$sth->fetchrow()){
    print "$rpfits_path/$rpfits_file\n";
    push @files_to_process,"$rpfits_path/$rpfits_file";
    push @just_files,$rpfits_file;
}
$sth->finish;

for (my $i=0;$i<=$#files_to_process;$i++){
    # get some info about calibration events, paddles and pointings
    # now we use rpfhdr to get some info about the events that
    # this FITS file describes. events are things like paddles,
    # and calibration commands
    my $eventcolour_acal=2;
    my $eventcolour_dcal=3;
    my $eventcolour_pcal=4;
    my @eventtime;
    my @eventcolour;
    my $obslog_string="";
    my $command="rpfhdr ";
    $command.=$files_to_process[$i];
    open(EVENTS,"-|")||exec $command;
    my $currheader;
    my ($event_type,$event_time,$acal_if1_value,$acal_if2_value,
	$event_source,$if1_freq,$if2_freq,$project,$rpfits_file);
    $rpfits_file=$just_files[$i];
    while(<EVENTS>){
	chomp;
	my $line=$_;
	my @els=split(/\s+/,$line);
	if ($line=~/^RPFITS header number/){
	    $currheader=$els[3];
	} elsif ($line=~/^OBSLOG/){
	    $obslog_string.=$line."\n";
	    # look for calibration commands
	    # get some info from the header
	    if (($line=~/acal/)||($line=~/pcal/)||($line=~/dcal/)){
		my $was_applied=0;
		$event_time=$els[1]." ".$els[2];
		if ($line=~/acal/){
		    $event_type="acal";
		    # only continue if there is an a at the end
		    # ie. the calibration was applied
		    if ($els[$#els] eq "a"){
			$was_applied=1;
		    }
		} elsif ($line=~/pcal/){
		    $event_type="pcal";
		    if (($els[$#els] eq "a")&&($els[$#els-1] eq "pcal")){
			$was_applied=1;
		    }
		} elsif ($line=~/dcal/){
		    $event_type="dcal";
		    if (($els[$#els] eq "a")&&($els[$#els-1] eq "dcal")){
			$was_applied=1;
		    }
		}
		if ($was_applied==1){
		    push @eventtime,&parse_miriad_time($event_time);
		    if ($event_type eq "acal"){
			push @eventcolour,$eventcolour_acal;
		    } elsif ($event_type eq "pcal"){
			push @eventcolour,$eventcolour_pcal;
		    } elsif ($event_type eq "dcal"){
			push @eventcolour,$eventcolour_dcal;
		    }
		}
	    }
	}
    }
    close(EVENTS);

    my $cycletime;
    # atlod time
    # luckily we don't care about anything special, we just want to
    # know what's in the file
    my $atlod_command="atlod in=$files_to_process[$i] out=$staging_area/tmp_osp.uv";
    system $atlod_command;
    
    # get the integration time from the file
    my $uvlist_command="uvlist vis=$staging_area/tmp_osp.uv options=full,variables log=tmp_osp.log";
    system $uvlist_command;
    open(UVLIST,"tmp_osp.log");
    my $cont_act=0;
    while(<UVLIST>){
	chomp;
	my @els=split(/\s+/);
	if ($els[0] eq "inttime"){
	    $cycletime=$els[2];
	    $cont_act=1;
	}
    }
    close(UVLIST);
    system "rm tmp_osp.log";

    if ($cont_act==1){
	# generate the uvindex log
	my $interval=$cycletime/60.0;
	my $uvindex_command="uvindex vis=$staging_area/tmp_osp.uv interval=".$interval." log=uvindex.log";
	system $uvindex_command;
	
	# read the uvindex log
	open(UVINDEX,"uvindex.log");
	my @date_time;
	my @source;
	my @calcode;
	my @nants;
	my @channels;
	my @freqconfig;
	my $in_description=0;
	my $file_description_string="";
	while(<UVINDEX>){
	    chomp;
	    my $line=$_;
	    if ($line=~/^\d{2}\D{3}\d{2}\:\d{2}\:\d{2}\:\d{2}\.\d/){
		# what we want
		my @els=split(/\s+/,$line);
		push @date_time,$els[0];
		push @source,$els[1];
		push @calcode,$els[2];
		push @nants,$els[3];
		push @channels,$els[4];
		push @freqconfig,$els[6];
	    } elsif ($line eq "------------------------------------------------"){
		# now comes the description of what the
		# file contains
		$in_description=1;
	    }
	    if ($in_description==1){
		$file_description_string.="$line\n";
	    }
	}
	close(UVINDEX);
	system "rm uvindex.log";
	system "rm -rf $staging_area/tmp_osp.uv";
	
	# start the plot
	my $pgplot_device="$just_files[$i]_summary.png/png";
	pgopen($pgplot_device);
	
	# set the foreground and background colours
	pgscr(0,1.0,1.0,1.0);
	pgscr(1,0.0,0.0,0.0);
	
	# find the first and last times
	my $first_time=&parse_miriad_time($date_time[0]);
	my $last_time=&parse_miriad_time($date_time[0]);
	for (my $i=1;$i<=$#date_time;$i++){
	    my $current_time=&parse_miriad_time($date_time[$i]);
	    if ($current_time<$first_time){
		$first_time=$current_time;
	    }
	    if ($current_time>$last_time){
		$last_time=$current_time;
	    }
	}
	
	# find how many configs there are
	my $current_source=$source[0];
	my $current_calcode=$calcode[0];
	my $current_nants=$nants[0];
	my $current_channels=$channels[0];
	my $current_freqconfig=$freqconfig[0];
	my $nconfigs=1;
	my @tconfigs;
	for (my $j=1;$j<=$#date_time;$j++){
	    if (($source[$j] ne $current_source)||
		($calcode[$j] ne $current_calcode)||
		($nants[$j] != $current_nants)||
		($channels[$j] != $current_channels)||
		($freqconfig[$j] != $current_freqconfig)){
		my $configstring="$current_source/$current_calcode/$current_nants/$current_channels/$current_freqconfig";
		my $isnew=1;
		for (my $k=0;$k<=$#tconfigs;$k++){
		    if ($tconfigs[$k] eq $configstring){
			$isnew=0;
		    }
		}
		if ($isnew==1){
		    push @tconfigs,$configstring;
		    $nconfigs++;
		}
		$current_source=$source[$j];
		$current_calcode=$calcode[$j];
		$current_nants=$nants[$j];
		$current_channels=$channels[$j];
		$current_freqconfig=$freqconfig[$j];
	    }
	}
	my $min_y_value=0;
	my $max_y_value=100;
	my $y_interval=($max_y_value-$min_y_value)/$nconfigs;
	
	# now set up the axes
	pgswin($first_time,$last_time,$min_y_value,$max_y_value);
	pgtbox('BCNTSZ',0,0,'BC',0,0);
	pgsah(2,70.0,1.0);
	pglab("Time","Configuration (source/calcode/nants/nchans/freqconfig)","Observation summary for ".$just_files[$i]);
	
	# now make the plot
	my $start_time=$first_time;
	my $start_time_string=$date_time[0];
	my $end_time=$first_time;
	my $end_time_string=$date_time[0];
	$current_source=$source[0];
	$current_calcode=$calcode[0];
	$current_nants=$nants[0];
	$current_channels=$channels[0];
	$current_freqconfig=$freqconfig[0];
	my $current_y=$max_y_value-$y_interval;
	my @configs;
	my @config_ys;
	my @config_earlytime;
	my @config_latetime;
	for (my $j=1;$j<=$#date_time;$j++){
	    # first find the time this source/config ends
	    if (($source[$j] ne $current_source)||
		($calcode[$j] ne $current_calcode)||
		($nants[$j] != $current_nants)||
		($channels[$j] != $current_channels)||
		($freqconfig[$j] != $current_freqconfig)){
		# something has changed
		$end_time=&parse_miriad_time($date_time[$j-1]);
		$end_time_string=$date_time[$j-1];
		# make the config string
		my $configstring="$current_source/$current_calcode/$current_nants/$current_channels/$current_freqconfig";
		print "$configstring: $start_time_string -> $end_time_string\n";
		# where to put it
		my $yheight=$current_y;
		my $found_config=0;
		for (my $k=0;$k<=$#configs;$k++){
		    if ($configs[$k] eq $configstring){
			$found_config=1;
			$yheight=$config_ys[$k];
			if ($start_time<$config_earlytime[$k]){
			    $config_earlytime[$k]=$start_time;
			}
			if ($end_time>$config_latetime[$k]){
			    $config_latetime[$k]=$end_time;
			}
		    }
		}
		if ($found_config==0){
		    push @configs,$configstring;
		    push @config_ys,$current_y;
		    push @config_earlytime,$start_time;
		    push @config_latetime,$end_time;
		    $current_y-=$y_interval;
		}
		# make a double ended arrow
		pgsch(0.2);
		pgarro($start_time,$yheight,$end_time,$yheight);
		pgarro($end_time,$yheight,$start_time,$yheight);
		# continue with the new config
		$start_time=&parse_miriad_time($date_time[$j]);
		$start_time_string=$date_time[$j];
		$current_source=$source[$j];
		$current_calcode=$calcode[$j];
		$current_nants=$nants[$j];
		$current_channels=$channels[$j];
		$current_freqconfig=$freqconfig[$j];
	    }
	}
	
	for (my $j=0;$j<=$#configs;$j++){
	    pgsch(0.6);
	    my ($configxlength,$configylength);
	    pglen(4,$configs[$j],$configxlength,$configylength);
	    my $startdiff=$config_earlytime[$j]-$first_time;
	    my $enddiff=$last_time-$config_latetime[$j];
	    my $leftxpos;
	    if ((($startdiff==0)&&($enddiff==0))||
		(($startdiff<$configxlength)&&($enddiff<$configxlength))){
		# need to put it above the line
		$leftxpos=($config_latetime[$j]-$config_earlytime[$j])/2.0+
		    $config_earlytime[$j];
		pgptxt($leftxpos,$config_ys[$j]+1,0.0,0.5,$configs[$j]);
	    } else {
		if ($startdiff>$enddiff){
		    $leftxpos=$config_earlytime[$j]-$configxlength-120;
		} else {
		    $leftxpos=$config_latetime[$j]+120;
		}
		pgtext($leftxpos,$config_ys[$j],$configs[$j]);
	    }
	}

	# add event lines
	for (my $j=0;$j<=$#eventtime;$j++){
	    my @xpoints;
	    my @ypoints;
	    $xpoints[0]=$eventtime[$j];
	    $xpoints[1]=$eventtime[$j];
	    $ypoints[0]=$min_y_value;
	    $ypoints[1]=$max_y_value;
	    pgsls(2);
	    pgsci($eventcolour[$j]);
	    pgline(2,\@xpoints,\@ypoints);
	}
	
	pgclos();
	
	print $file_description_string;
	
	# put everything into the database table rpfits_summary
	my $scp_command="scp -P 2222 $just_files[$i]_summary.png postgres\@localhost:.";
	print "copying with $scp_command\n";
	system $scp_command;
	my $insi_query="INSERT INTO rpfits_summary VALUES('".$just_files[$i]."',".
	    "'".$file_description_string."',lo_import('/var/lib/postgresql/".$just_files[$i].
	    "_summary.png'),'$obslog_string')";
	print "$insi_query\n";
	my $insi=$dbh->prepare($insi_query);
	$insi->execute;
	$insi->finish;
	
	# and do some cleaning up
	my $clean_command="ssh -p 2222 postgres\@localhost rm $just_files[$i]_summary.png";
	system $clean_command;
	$clean_command="rm $just_files[$i]\_summary.png";
	system $clean_command;

    } else {
	system "rm uvindex.log";
	system "rm -rf $staging_area/tmp_osp.uv";
    }
    
    # and now mark this file as summarised
    my $urp_query="UPDATE rpfits_files SET summarised=TRUE WHERE file='".
	$just_files[$i]."'";
    my $urp=$dbh->prepare($urp_query);
    $urp->execute;
    $urp->finish;
    if ($cont_act==0){
	# there was no data in this file
	$urp_query="UPDATE rpfits_files SET nodata=TRUE WHERE file='".
	    $just_files[$i]."'";
	$urp=$dbh->prepare($urp_query);
	$urp->execute;
	$urp->finish;
    }
}

$dbh->disconnect;

sub parse_miriad_time {
    my $timestring=$_[0];
    my $bdt_month;
    my $bdt_year;
    if ($timestring=~/(\d{2})(\D{3})(\d{2})\:(\d{2})\:(\d{2})\:(\d{2}\.\d)/){
	$timestring=~m/(\d{2})(\D{3})(\d{2})\:(\d{2})\:(\d{2})\:(\d{2}\.\d)/;
	$bdt_month=&month_parse($2);
	$bdt_year=$1+100;
    } elsif ($timestring=~/(\d{4})-(\d{2})-(\d{2})\s(\d{2})\:(\d{2})\:(\d{2})/){
	$timestring=~m/(\d{4})-(\d{2})-(\d{2})\s(\d{2})\:(\d{2})\:(\d{2})/;
	$bdt_month=$2-1;
	$bdt_year=$1-1900;
    }
    my $bdt_date=$3;
    my $bdt_hour=$4;
    my $bdt_minute=$5;
    my $bdt_second=$6;
    if ($bdt_second>=60){
	$bdt_minute+=1;
	$bdt_second-=60;
    }
    my $simpletime=timegm($bdt_second,$bdt_minute,$bdt_hour,$bdt_date,$bdt_month,$bdt_year);
    # get the day of year
    my ($s,$m,$h,$md,$mo,$y,$w,$yd,$id)=gmtime($simpletime);
    # get the number of seconds since the start of the day
    while ($simpletime>86400){
	$simpletime-=86400;
    }
    # and add the day of year
    $simpletime+=$yd*86400;

    return $simpletime;
}

sub month_parse {
    my $month_string=$_[0];
    my $month_number;
    if ($month_string eq "JAN"){
	$month_number=0;
    } elsif ($month_string eq "FEB"){
	$month_number=1;
    } elsif ($month_string eq "MAR"){
	$month_number=2;
    } elsif ($month_string eq "APR"){
	$month_number=3;
    } elsif ($month_string eq "MAY"){
	$month_number=4;
    } elsif ($month_string eq "JUN"){
	$month_number=5;
    } elsif ($month_string eq "JUL"){
	$month_number=6;
    } elsif ($month_string eq "AUG"){
	$month_number=7;
    } elsif ($month_string eq "SEP"){
	$month_number=8;
    } elsif ($month_string eq "OCT"){
	$month_number=9;
    } elsif ($month_string eq "NOV"){
	$month_number=10;
    } elsif ($month_string eq "DEC"){
	$month_number=11;
    }
    return $month_number;
}

sub combine_elements {
    my @els=@_;
    my $retstring;
    $retstring=$els[5];
    for (my $i=6;$i<=$#els;$i++){
	$retstring.=" ".$els[$i];
    }
    return $retstring;
}
